//package com.batch_demo;
//
//import com.alibaba.fastjson.JSONArray;
//import com.alibaba.fastjson.JSONObject;
//import com.batch_demo.util.ExportExcel;
//import com.batch_demo.util.HttpClientUtil;
//import com.batch_demo.util.IpUtil;
//import org.jsoup.nodes.Document;
//import org.jsoup.nodes.Element;
//import org.jsoup.nodes.Node;
//import org.jsoup.nodes.TextNode;
//import org.jsoup.select.Elements;
//
//import java.io.FileOutputStream;
//import java.io.PrintStream;
//import java.util.ArrayList;
//import java.util.HashMap;
//import java.util.List;
//import java.util.Map;
//
//public class BatchDemo {
//    public void getListPage() throws Exception {
//        String url = "https://www.tianyancha.com/search/p1?key=%E5%B9%BF%E5%91%8A";
//        int pn = 1;
//        int nn = 10;
//        url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=con&from=webmap&c=289&wd=%E5%B9%BF%E5%91%8A%E5%85%AC%E5%8F%B8&wd2=&pn=16&nn=160&db=0&sug=0&addr=0&&da_src=pcmappg.poi.page&on_gel=1&src=7&gr=3&l=11&tn=B_NORMAL_MAP&u_loc=13517343,3639273&ie=utf-8&b=(13462721.31,3606682.64;13596097.31,3692570.64)&t=1505114151041";
//        //上海
//        url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=con&from=webmap&c=289&wd=%E5%B9%BF%E5%91%8A%E5%85%AC%E5%8F%B8&wd2=&pn=16&nn=160&db=0&sug=0&addr=0&&da_src=pcmappg.poi.page&on_gel=1&src=7&gr=3&l=11&tn=B_NORMAL_MAP&u_loc=13517343,3639273&ie=utf-8&b=(12938630.559999997,4803875.470000006;13005318.559999997,4846819.470000006)&t=1505120654943";
//        //北京
//        url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=direct&pcevaname=pc4.1&qt=con&from=webmap&c=131&wd=%E5%B9%BF%E5%91%8A%E5%85%AC%E5%8F%B8&wd2=&pn=1&nn=10&db=0&sug=0&addr=0&&da_src=pcmappg.poi.page&on_gel=1&src=7&gr=3&l=12&tn=B_NORMAL_MAP&u_loc=13517343,3639273&ie=utf-8&b=(12938630.559999997,4803875.470000006;13005318.559999997,4846819.470000006)&t=1505121481664";
//        List<Object[]> dataList = new ArrayList<>();
//        while (pn < 5000) {
//            if (pn % 100 == 0) {
//                dataList = new ArrayList<>();
//            }
//            try {
//                String u = url.replace("pn=1", "pn=" + pn).replace("nn=10", "nn=" + (pn * nn));
//                JSONObject result = HttpClientUtil.getJson(u);
//                if (null == result) {
//                    export(dataList, String.valueOf(pn));
//                    break;
//                }
//                JSONArray companyList = result.getJSONArray("content");
//                if (null != companyList) {
//                    for (int i = 0; i < companyList.size(); i++) {
//                        Object[] objects = new Object[5];
//                        JSONObject company = companyList.getJSONObject(i);
//                        String addr = company.getString("addr");
//                        objects[0] = addr;
//                        String areaName = company.getString("area_name");
//                        objects[1] = areaName;
//                        String area = company.getString("area");
//                        objects[2] = area;
//                        String name = company.getString("name");
//                        objects[3] = name;
//                        String tel = company.getString("tel");
//                        objects[4] = tel;
//                        dataList.add(objects);
//                    }
//                } else {
//                    export(dataList, String.valueOf(pn));
//                    break;
//                }
//            } catch (Exception e) {
//                e.printStackTrace();
//                export(dataList, String.valueOf(pn));
//            }
//            if (pn % 100 == 0) {
//                export(dataList, String.valueOf(pn));
//            }
//            pn++;
//            Thread.sleep(3000);
//        }
//
//    }
//
//    private void export(List<Object[]> dataList, String pn) throws Exception {
//        String title = "广告公司" + pn;
////        String[] rowName = {"addr", "areaName", "area", "name", "tel"};
//        String[] rowName = {"name", "address", "telephone", "tag", "type"};
//
//        ExportExcel exportExcel = new ExportExcel(title, rowName, dataList);
//        FileOutputStream fileOutputStream = new FileOutputStream(pn + ".xls");
//        exportExcel.export(fileOutputStream);
//    }
//
//    public void getPOIListByBaiduMap() throws Exception {
//        String[] regionArr = {"上海", "北京", "合肥", "安庆", "蚌埠", "亳州", "巢湖", "池州", "滁州", "阜阳", "淮北", "淮南", "黄山", "六安", "马鞍山", "宿州", "铜陵", "芜湖", "宣城", "福州", "龙岩", "南平", "宁德", "莆田", "泉州", "三明", "厦门", "漳州", "广州", "潮州", "东莞", "佛山", "河源", "惠州", "江门", "揭阳", "茂名", "梅州", "清远", "汕头", "汕尾", "韶关", "深圳", "阳江", "云浮", "湛江", "肇庆", "中山", "珠海", "南宁", "百色", "北海", "崇左", "防城港", "桂林", "贵港", "河池", "贺州", "来宾", "柳州", "钦州", "梧州", "玉林", "贵阳", "安顺", "毕节市", "六盘水", "铜仁市", "遵义", "黔西南布依族苗族自治州", "黔东南苗族侗族自治州", "黔南布依族苗族自治州", "兰州", "白银", "定西", "甘南藏族自治州", "嘉峪关", "金昌", "酒泉", "临夏回族自治州", "陇南", "平凉", "庆阳", "天水", "武威", "张掖", "海口", "白沙黎族自治县", "保亭黎族苗族自治县", "昌江黎族自治县", "儋州", "澄迈", "东方", "定安", "琼海", "琼中黎族苗族自治县", "乐东黎族自治县", "临高", "陵水黎族自治县", "三亚", "屯昌", "万宁", "文昌", "五指山", "郑州", "安阳", "鹤壁", "焦作", "开封", "洛阳", "漯河", "南阳", "平顶山", "濮阳", "三门峡", "商丘", "新乡", "信阳", "许昌", "周口", "驻马店", "哈尔滨", "大庆", "大兴安岭地区", "鹤岗", "黑河", "鸡西", "佳木斯", "牡丹江", "七台河", "齐齐哈尔", "双鸭山", "绥化", "伊春", "武汉", "鄂州", "恩施土家族苗族自治州", "黄冈", "黄石", "荆门", "荆州", "潜江", "神农架林区", "十堰", "随州", "天门", "仙桃", "咸宁", "襄阳", "孝感", "宜昌", "长沙", "常德", "郴州", "衡阳", "怀化", "娄底", "邵阳", "湘潭", "湘西土家族苗族自治州", "益阳", "永州", "岳阳", "张家界", "株洲", "石家庄", "保定", "沧州", "承德", "邯郸", "衡水", "廊坊", "秦皇岛", "唐山", "邢台", "张家口", "南京", "常州", "淮安", "连云港", "南通", "苏州", "宿迁", "泰州", "无锡", "徐州", "盐城", "扬州", "镇江", "南昌", "抚州", "赣州", "吉安", "景德镇", "九江", "萍乡", "上饶", "新余", "宜春", "鹰潭", "长春", "白城", "白山", "吉林市", "辽源", "四平", "松原", "通化", "延边朝鲜族自治州", "沈阳", "鞍山", "本溪", "朝阳", "大连", "丹东", "抚顺", "阜新", "葫芦岛", "锦州", "辽阳", "盘锦", "铁岭", "营口", "银川", "固原", "石嘴山", "吴忠", "中卫", "呼和浩特", "阿拉善盟", "包头", "巴彦淖尔", "赤峰", "鄂尔多斯", "呼伦贝尔", "通辽", "乌海", "乌兰察布", "锡林郭勒盟", "兴安盟", "西宁", "果洛藏族自治州", "海东市", "海北藏族自治州", "海南藏族自治州", "海西蒙古族藏族自治州", "黄南藏族自治州", "玉树藏族自治州", "济南", "滨州", "东营", "德州", "菏泽", "济宁", "莱芜", "聊城", "临沂", "青岛", "日照", "泰安", "威海", "潍坊", "烟台", "枣庄", "淄博", "太原", "长治", "大同", "晋城", "晋中", "临汾", "吕梁", "朔州", "忻州", "阳泉", "运城", "西安", "安康", "宝鸡", "汉中", "商洛", "铜川", "渭南", "咸阳", "延安", "榆林", "成都", "阿坝藏族羌族自治州", "巴中", "达州", "德阳", "甘孜藏族自治州", "广安", "广元", "乐山", "凉山彝族自治州", "泸州", "南充", "眉山", "绵阳", "内江", "攀枝花", "遂宁", "雅安", "宜宾", "资阳", "自贡", "台北", "高雄", "台中", "台南", "新北", "基隆", "新竹", "嘉义", "拉萨", "阿里地区", "昌都市", "林芝市", "那曲地区", "日喀则市", "山南地区", "乌鲁木齐", "阿拉尔", "阿克苏地区", "阿勒泰地区", "巴音郭楞蒙古自治州", "博尔塔拉蒙古自治州", "昌吉回族自治州", "哈密地区", "和田地区", "喀什地区", "克拉玛依", "克孜勒苏柯尔克孜自治州", "石河子", "塔城地区", "图木舒克", "吐鲁番地区", "五家渠", "伊犁哈萨克自治州", "昆明", "保山", "楚雄彝族自治州", "大理白族自治州", "德宏傣族景颇族自治州", "迪庆藏族自治州", "红河哈尼族彝族自治州", "丽江", "临沧", "怒江傈僳族自治州", "普洱", "曲靖", "昭通", "文山壮族苗族自治州", "西双版纳傣族自治州", "玉溪", "杭州", "湖州", "嘉兴", "金华", "丽水", "宁波", "衢州", "绍兴", "台州", "温州", "舟山", "香港", "澳门"};
//        String url = "http://api.map.baidu.com/place/v2/search?query=广告公司&page_size=10&page_num=0&scope=2&region=上海&output=json&ak=lBajF1g0iEotrE2mkyDZ2fAgH2AorAXu";
//
//        int index = 0;
//        for (String region : regionArr) {
//            index++;
//            int pageNum = 0;
//            List<Object[]> dataList = new ArrayList<>();
//
//            while (pageNum < 1000) {
//                if (pageNum % 100 == 0) {
//                    dataList = new ArrayList<>();
//                }
//                String u = url.replace("page_num=0", "page_num=" + pageNum).replace("上海", region);
//                System.out.println(u);
//                JSONObject jsonObject = HttpClientUtil.getJson(u);
//                if (!jsonObject.getString("status").equals("0")) {
//                    export(dataList, index + "_" + region + "_" + pageNum);
//                    break;
//                }
//                JSONArray results = jsonObject.getJSONArray("results");
//                if (null == results || results.size() < 1) {
//                    export(dataList, index + "_" + region + "_" + pageNum);
//                    break;
//                }
//                for (int i = 0; i < results.size(); i++) {
//                    Object[] objects = new Object[5];
//                    JSONObject obj = results.getJSONObject(i);
//                    objects[0] = obj.getString("name");
//                    objects[1] = obj.getString("address");
//                    objects[2] = obj.getString("telephone");
//                    JSONObject detailInfo = obj.getJSONObject("detail_info");
//                    objects[3] = detailInfo.getString("tag");
//                    objects[4] = detailInfo.getString("type");
//                    dataList.add(objects);
//                }
//                pageNum++;
//                if (pageNum % 100 == 0) {
//                    export(dataList, index + "_" + region + "_" + pageNum);
//                    continue;
//                }
//                //export(dataList, region+"_"+pageNum);
//            }
//        }
//    }
//
//    public void getPOIDetailByBaiduMap() {
//        String url = "http://api.map.baidu.com/place/v2/detail?uid=5a8fb739999a70a54207c130&output=json&scope=2&ak=lBajF1g0iEotrE2mkyDZ2fAgH2AorAXu";
//    }
//
//    public void getHuangye() throws Exception {
//        String[] keywords = {"广告", "传媒", "媒体", "平面", "传播", "策划", "会展", "平面设计"};
//        String url = "http://www.huangye88.com/search.html?kw=%E5%B9%BF%E5%91%8A&type=company";
//        for (String keyword : keywords) {
//            int pageNum = 1;
//            List<Object[]> dataList = new ArrayList<>();
//            int index = 0;
//            while (pageNum < 10000) {
//                index++;
//                String u = url.replace("广告", keyword);
//                if (pageNum != 1) {
//                    u = url + "&page=" + pageNum;
//                }
//
//                Map<String, String> map = new HashMap<>();
//                map.put("kw", keyword);
//                map.put("cattrace", "");
//                map.put("type", "company");
//                map.put("pageType", "13");
//                map.put("a", "0");
//                map.put("c", "0");
//                map.put("u", "0");
//
//                Map<String, String> headerMap = new HashMap<>();
//                headerMap.put("Cookie", "PHPSESSID=c892a6608a834f408292d7d572ca005259b8f2d25656b0.95111886; Hm_lvt_c8184fd80a083199b0e82cc431ab6740=1505293029,1506265901,1506396944; Hm_lpvt_c8184fd80a083199b0e82cc431ab6740=1506418061; _ga=GA1.2.1241460307.1505293029; _gid=GA1.2.302250075.1506396945");
//                headerMap.put("Upgrade-Insecure-Requests", "1");
//                headerMap.put("Content-Type", "application/x-www-form-urlencoded");
//                headerMap.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36");
//                headerMap.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
//                headerMap.put("Referer", u);
//                Document document = HttpClientUtil.postHtml(u, map, headerMap);
//                Elements elements = document.body().select("body div.middle div.mid-cont div.content ul");
//                if (null == elements || elements.size() < 1) {
//                    export(dataList, index + "_" + keyword + "_" + pageNum);
//                    return;
//                }
//                for (Element element : elements) {
//                    List<Node> nodes = element.childNodes();
//                    for (Node node : nodes) {
//                        if (node instanceof TextNode) {
//                            continue;
//                        }
//                        Element nodeElement = (Element) node;
//                        List<Node> nodes1 = nodeElement.childNodes();
//                        for (Node n : nodes1) {
//                            if (n instanceof TextNode) {
//                                continue;
//                            }
//                            Element nElement = (Element) n;
//                            nElement.getElementsByIndexEquals(1);
//                            System.out.println(nElement);
//                        }
//                    }
//                }
//                pageNum++;
//            }
//        }
//    }
//
//    private void getIqiyi() {
//        String url = "http://list.iqiyi.com/www/1/-------------8-{pageNum}-1-iqiyi--.html";
//
//
//        for (int i = 0; i < 100; i ++) {
//            try {
//                String u = url.replace("{pageNum}", String.valueOf(i + 1));
//                Map<String, String> headerMap = new HashMap<>();
//                headerMap.put("Cookie", "__uuid=faa90e76-234d-ccc9-4e34-ea3121182b2d; player_forcedType=h5_VOD; P00004=-1212058482.1499787410.263931a43d; QC005=18210fabdae7c2be02a3b6e111513812; _ga=GA1.2.986288847.1501293685; P00001=88LM2QK4XOVpp3fFJOOm2rFaoCNovQZUsnlvvffm2tGfwIlpEm25B9Q0T0hXZMJDUqzVD64; P00003=1428966571; P00010=1428966571; P01010=1501344000; P00007=88LM2QK4XOVpp3fFJOOm2rFaoCNovQZUsnlvvffm2tGfwIlpEm25B9Q0T0hXZMJDUqzVD64; P00PRU=1428966571; P00002=%7B%22uid%22%3A%221428966571%22%2C%22user_name%22%3A%2213621864617%22%2C%22email%22%3A%22%22%2C%22nickname%22%3A%22%5Cu7231%5Cu6253%5Cu7f51%5Cu7403%5Cu306e%5Cu949f%5Cu79bb%5Cu77e5%5Cu6167%22%2C%22pru%22%3A1428966571%2C%22type%22%3A11%2C%22pnickname%22%3A%22%5Cu7231%5Cu6253%5Cu7f51%5Cu7403%5Cu306e%5Cu949f%5Cu79bb%5Cu77e5%5Cu6167%22%7D; P000email=13621864617; QC116=; QIYUECK=qy_pc_f49169df1c974561b9297f966983226d; QC134=%7B%221428966571%22%3A%5B1501293839967%5D%7D; QP001=1; QC118=%7B%22color%22%3A%22FFFFFF%22%2C%22channelConfig%22%3A0%2C%22hadTip%22%3A1%7D; QC159=%7B%22color%22%3A%22FFFFFF%22%2C%22channelConfig%22%3A1%2C%22hadTip%22%3A1%2C%22isOpen%22%3A0%2C%22speed%22%3A10%2C%22density%22%3A30%2C%22opacity%22%3A68%2C%22isFilterColorFont%22%3A1%2C%22proofShield%22%3A0%2C%22forcedFontSize%22%3A24%2C%22isFilterImage%22%3A1%2C%22isset%22%3A1%7D; QC021=%5B%7B%22key%22%3A%22%E7%99%BD%E5%A4%9C%E8%BF%BD%E5%87%B6%22%7D%2C%7B%22key%22%3A%22%E7%88%B6%E5%AD%90%E9%9B%84%E5%85%B5%22%7D%2C%7B%22key%22%3A%22%E8%8C%83%E4%BC%9F%22%7D%5D; QC124=1%7C0; T00404=4aad44822d64a4f5f85bab26068dbf60; Hm_lvt_53b7374a63c37483e5dd97d78d9bb36e=1506598221,1507603114; T00700=EgcI4b-tIRAB; JSESSIONID=aaaRLZw1k0l9cbq_LVb6v; QC001=1; QC008=1496547164.1508225991.1508225991.15; c11428966571=1508225992575; c241428966571=1508225992575; QC007=DIRECT; __dfp=a038e3005919a24971b287ecf047c5bfd595e49408507229d2ca5e2ac44154a959@1510817992816@1508225992816; QC006=1pg0apizsbzzdiqbz03lwrb6; QC010=210075990; QC140=3");
//                headerMap.put("Pragma", "no-cache");
//                headerMap.put("Upgrade-Insecure-Requests", "1");
//                headerMap.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36");
//                headerMap.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
//                headerMap.put("Accept-Encoding", "gzip, deflate");
//                headerMap.put("Accept-Language", "zh-CN;zh;q=0.8,en;q=0.6");
//                headerMap.put("Cache-Control", "no-cache");
//                headerMap.put("Connection", "keep-alive");
//                headerMap.put("x-forwarded-for",IpUtil.getRandomIp());
//                System.err.println(u);
//                Document document = HttpClientUtil.getHtml(u, headerMap);
//                Elements elements = document.body().select("body div.page-list.page-list-type1 div div div.wrapper-cols div ul li");
//                for (Element element : elements) {
//                    Elements elements1 = element.select(" div.site-piclist_info div.mod-listTitle_left p a");
//                    for (Element element1 : elements1) {
//                        List<Node> nodes = element1.childNodes();
//                        for (Node node : nodes) {
//                            if (node instanceof TextNode) {
//                                System.out.println(((TextNode) node).getWholeText());
//                            }
//                        }
//                    }
//                }
//
//                try {
//                    Thread.sleep(3000);
//                } catch (InterruptedException e) {
//                    e.printStackTrace();
//                }
//            }catch (Exception e) {
//                System.err.println(i);
//            }
//        }
//
//    }
//
//    public static void main(String[] args) throws Exception {
//        PrintStream printStream = new PrintStream("data.txt");
//        System.setOut(printStream);
//        BatchDemo batchDemo = new BatchDemo();
////        batchDemo.getHuangye();
//        batchDemo.getIqiyi();
////        batchDemo.getListPage();
////        batchDemo.getPOIListByBaiduMap();
////        Thread thread = new Thread(() -> {
////            while (true) {
////                System.out.println();
////                try {
////                    Thread.sleep(1000);
////                } catch (InterruptedException e) {
////                    e.printStackTrace();
////                }
////            }
////        });
////
////        thread.start();
////
////        thread.join();
//    }
//}
